{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../code')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from psumm import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = Input(\n",
    "    [\"Georgia mayor assails governor's move to reopen beaches.\".lower(),\n",
    "    \"The Tybee Island city council voted to close the beaches on March 20.\".lower(),\n",
    "    \"But Georgia Gov. Brian Kemp issued a statewide shelter-in-place executive order which supersedes all local orders relating to coronavirus and also opened up the state's beaches.\".lower()\n",
    "    ],\n",
    "    [1, 0, 0]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "gi = dict(InfoF().gi_f(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "words1 = i.data[0].lower()[:-1].split(\" \")\n",
    "words2 = i.data[1].lower()[:-1].split(\" \")\n",
    "words3 = i.data[2].lower()[:-1].split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = Counter(words1 + words2 + words3)\n",
    "ci = Counter(words1)\n",
    "cd = Counter(words2 + words3)\n",
    "\n",
    "lens = len(words1 + words2 + words3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import log as ilog\n",
    "\n",
    "def log(x):\n",
    "    if x == 0: return 0\n",
    "    return ilog(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def smi(word, lamda = 0.1):\n",
    "    p_x = c[word] / lens\n",
    "    p_yi = len(words1) / lens\n",
    "    p_yd = len(words2 + words3) / lens\n",
    "\n",
    "    p_x_yi = ci[word] / lens\n",
    "    p_x_yd = cd[word] / lens\n",
    "    \n",
    "\n",
    "    #print(c[word], len(words1), len(words2 + words3), ci[word], cd[word], lens)\n",
    "    \n",
    "    #print(lamda * p_x_yd * log(p_x_yd / (p_x * p_yd)) )\n",
    "    \n",
    "    return \"%.4f\" % (p_x_yi * log(p_x_yi / (p_x * p_yi)) + lamda * p_x_yd * log(p_x_yd / (p_x * p_yd)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0039699139485194065\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'0.00'"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smi(\"tybee\", 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "governor's\n",
      "gov.\n",
      "a\n",
      "shelter-in-place\n",
      "state's\n",
      "('mayor', '0.0377', '0.6365', 1, 0, '0.0377')\n",
      "('assails', '0.0377', '0.6365', 1, 0, '0.0377')\n",
      "('move', '0.0377', '0.6365', 1, 0, '0.0377')\n",
      "('reopen', '0.0377', '0.6365', 1, 0, '0.0377')\n",
      "('georgia', '0.0132', '0.1744', 1, 1, '0.0121')\n",
      "('the', '0.0107', '0.6365', 0, 3, '0.0119')\n",
      "('to', '0.0059', '0.0000', 1, 2, '0.0050')\n",
      "('beaches', '0.0059', '0.0000', 1, 2, '0.0050')\n",
      "('tybee', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('island', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('city', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('council', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('voted', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('close', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('on', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('march', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('20', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('but', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('brian', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('kemp', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('issued', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('statewide', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('executive', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('order', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('which', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('supersedes', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('all', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('local', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('orders', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('relating', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('coronavirus', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('and', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('also', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('opened', '0.0036', '0.1744', 0, 1, '0.0040')\n",
      "('up', '0.0036', '0.1744', 0, 1, '0.0040')\n"
     ]
    }
   ],
   "source": [
    "#for word in [\"georgia\", \"mayor\", \"tybee\"]:\n",
    "res = []\n",
    "for word in c:\n",
    "    try:\n",
    "        x = (word, smi(word, 0.9), \"%.4f\" % gi[word], ci[word], cd[word], smi(word, 1))\n",
    "        res.append(x)\n",
    "    except:\n",
    "        print(word)\n",
    "        \n",
    "res = sorted(res, key=lambda x: x[1], reverse=True)\n",
    "for a in res:\n",
    "    print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c['beaches']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ci['beaches']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ci)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "governor's\n",
      "gov.\n",
      "a\n",
      "shelter-in-place\n",
      "state's\n",
      "('georgia', '0.0177', '0.1744', 2, 0, '0.0177')\n",
      "('mayor', '0.0089', '0.6365', 1, 0, '0.0089')\n",
      "('assails', '0.0089', '0.6365', 1, 0, '0.0089')\n",
      "('move', '0.0089', '0.6365', 1, 0, '0.0089')\n",
      "('reopen', '0.0089', '0.6365', 1, 0, '0.0089')\n",
      "('but', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('brian', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('kemp', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('issued', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('statewide', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('executive', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('order', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('which', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('supersedes', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('all', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('local', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('orders', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('relating', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('coronavirus', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('and', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('also', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('opened', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('up', '0.0089', '0.1744', 1, 0, '0.0089')\n",
      "('the', '-0.0349', '0.6365', 1, 2, '0.0263')\n",
      "('tybee', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('island', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('city', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('council', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('voted', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('close', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('on', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('march', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('20', '-0.0145', '0.1744', 0, 1, '0.0290')\n",
      "('to', '-0.0024', '0.0000', 2, 1, '0.0061')\n",
      "('beaches', '-0.0024', '0.0000', 2, 1, '0.0061')\n"
     ]
    }
   ],
   "source": [
    "words1 = i.data[0].lower()[:-1].split(\" \")\n",
    "words2 = i.data[1].lower()[:-1].split(\" \")\n",
    "words3 = i.data[2].lower()[:-1].split(\" \")\n",
    "\n",
    "c = Counter(words1 + words2 + words3)\n",
    "ci = Counter(words1+ words3)\n",
    "cd = Counter(words2 )\n",
    "\n",
    "lens = len(words1 + words2 + words3)\n",
    "\n",
    "\n",
    "def smi(word, lamda = 0.1):\n",
    "    p_x = c[word] / lens\n",
    "    p_yi = len(ci) / lens\n",
    "    p_yd = len(cd) / lens\n",
    "\n",
    "    p_x_yi = ci[word] / lens\n",
    "    p_x_yd = cd[word] / lens\n",
    "    \n",
    "\n",
    "    #print(c[word], len(words1), len(words2 + words3), ci[word], cd[word], lens)\n",
    "    \n",
    "    #print(lamda * p_x_yd * log(p_x_yd / (p_x * p_yd)) )\n",
    "    \n",
    "    return \"%.4f\" % (p_x_yi * log(p_x_yi / (p_x * p_yi)) + lamda * p_x_yd * log(p_x_yd / (p_x * p_yd)))\n",
    "\n",
    "\n",
    "#for word in [\"georgia\", \"mayor\", \"tybee\"]:\n",
    "res = []\n",
    "for word in c:\n",
    "    try:\n",
    "        x = (word, smi(word, -.5), \"%.4f\" % gi[word], ci[word], cd[word], smi(word, 1))\n",
    "        res.append(x)\n",
    "    except:\n",
    "        print(word)\n",
    "        \n",
    "res = sorted(res, key=lambda x: x[1], reverse=True)\n",
    "for a in res:\n",
    "    print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sln-summarizer",
   "language": "python",
   "name": "sln-summarizer"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
